import numpy as np
import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()
import pandas as pd
from tensorflow import keras
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot
import plotly.graph_objects as go
import math
import seaborn as sns
from sklearn.metrics import mean_squared_error

np.random.seed(1)
tf.random.set_seed(1)

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, GRU, Dropout, RepeatVector, TimeDistributed
from keras import backend

MODELFILENAME = 'MODELS/GRU_1d_TFM'
TIME_STEPS=144 #1d

CMODEL = GRU
MODEL = "GRU"
UNITS=55
DROPOUT=0.118
ACTIVATION='tanh'
OPTIMIZER='adamax'

EPOCHS=36
BATCHSIZE=9
VALIDATIONSPLIT=0.2

Càrrega de les dades

# Code to read csv file into Colaboratory:
# from google.colab import files
# uploaded = files.upload()
# import io
# df = pd.read_csv(io.BytesIO(uploaded['SentDATA.csv']))
# Dataset is now stored in a Pandas Dataframe
df = pd.read_csv('../../data/dadesTFM.csv')

df.reset_index(inplace=True)
df['Time'] = pd.to_datetime(df['Time'])
df = df.set_index('Time')

Transformació de dades

columns = ['PM1','PM25','PM10','PM1ATM','PM25ATM','PM10ATM']

df1 = df.copy();

df1 = df1.rename(columns={"PM 1":"PM1","PM 2.5":"PM25","PM 10":"PM10","PM 1 ATM":"PM1ATM","PM 2.5 ATM":"PM25ATM","PM 10 ATM":"PM10ATM"})

df1['PM1'] = df['PM 1'].astype(np.float32)
df1['PM25'] = df['PM 2.5'].astype(np.float32)
df1['PM10'] = df['PM 10'].astype(np.float32)
df1['PM1ATM'] = df['PM 1 ATM'].astype(np.float32)
df1['PM25ATM'] = df['PM 2.5 ATM'].astype(np.float32)
df1['PM10ATM'] = df['PM 10 ATM'].astype(np.float32)

df2 = df1.copy()

Crear dades d'entrenament i de test

train_size = int(len(df2) * 0.8)
test_size = len(df2) - train_size
train, test = df2.iloc[0:train_size], df2.iloc[train_size:len(df2)]
train.shape, test.shape

((3117, 7), (780, 7))

Normalitzar les dades d'entrenament

#Standardize the data
for col in columns:
    scaler = StandardScaler()
    train[col] = scaler.fit_transform(train[[col]])

<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  train[col] = scaler.fit_transform(train[[col]])
<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  train[col] = scaler.fit_transform(train[[col]])
<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  train[col] = scaler.fit_transform(train[[col]])
<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  train[col] = scaler.fit_transform(train[[col]])
<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  train[col] = scaler.fit_transform(train[[col]])
<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  train[col] = scaler.fit_transform(train[[col]])

Crear finestra de temps

def create_sequences(X, y, time_steps=TIME_STEPS):
    Xs, ys = [], []
    for i in range(len(X)-time_steps):
        Xs.append(X.iloc[i:(i+time_steps)].values)
        ys.append(y.iloc[i+time_steps])
    
    return np.array(Xs), np.array(ys)

X_train, y_train = create_sequences(train[[columns[1]]], train[columns[1]])
#X_test, y_test = create_sequences(test[[columns[1]]], test[columns[1]])

print(f'X_train shape: {X_train.shape}')
print(f'y_train shape: {y_train.shape}')

X_train shape: (2973, 144, 1)
y_train shape: (2973,)

Creació del model

#afegir nova mètrica
def rmse(y_true, y_pred):
	return backend.sqrt(backend.mean(backend.square(y_pred - y_true), axis=-1))

model = Sequential()
model.add(CMODEL(units = UNITS, return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])))
model.add(Dropout(rate=DROPOUT))
model.add(TimeDistributed(Dense(1,kernel_initializer='normal',activation=ACTIVATION)))
model.compile(optimizer=OPTIMIZER, loss='mae',metrics=['mse',rmse])
model.summary()

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
gru (GRU)                    (None, 144, 55)           9570      
_________________________________________________________________
dropout (Dropout)            (None, 144, 55)           0         
_________________________________________________________________
time_distributed (TimeDistri (None, 144, 1)            56        
=================================================================
Total params: 9,626
Trainable params: 9,626
Non-trainable params: 0
_________________________________________________________________

history = model.fit(X_train, y_train, epochs=EPOCHS, batch_size=BATCHSIZE, validation_split=VALIDATIONSPLIT,
                    callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, mode='min')], shuffle=False)

Epoch 1/36
265/265 [==============================] - 17s 64ms/step - loss: 0.7232 - mse: 0.8770 - rmse: 0.7553 - val_loss: 0.3444 - val_mse: 0.3247 - val_rmse: 0.3985
Epoch 2/36
265/265 [==============================] - 16s 62ms/step - loss: 0.6730 - mse: 0.7776 - rmse: 0.7360 - val_loss: 0.3556 - val_mse: 0.3378 - val_rmse: 0.4142
Epoch 3/36
265/265 [==============================] - 20s 74ms/step - loss: 0.6674 - mse: 0.7679 - rmse: 0.7352 - val_loss: 0.3618 - val_mse: 0.3441 - val_rmse: 0.4223
Epoch 4/36
265/265 [==============================] - 20s 75ms/step - loss: 0.6647 - mse: 0.7645 - rmse: 0.7351 - val_loss: 0.3673 - val_mse: 0.3490 - val_rmse: 0.4287
Epoch 5/36
265/265 [==============================] - 17s 63ms/step - loss: 0.6625 - mse: 0.7621 - rmse: 0.7348 - val_loss: 0.3702 - val_mse: 0.3522 - val_rmse: 0.4327
Epoch 6/36
265/265 [==============================] - 17s 63ms/step - loss: 0.6608 - mse: 0.7602 - rmse: 0.7344 - val_loss: 0.3730 - val_mse: 0.3547 - val_rmse: 0.4360

import matplotlib.pyplot as plt
plt.plot(history.history['loss'], label='MAE Training loss')
plt.plot(history.history['val_loss'], label='MAE Validation loss')
plt.plot(history.history['mse'], label='MSE Training loss')
plt.plot(history.history['val_mse'], label='MSE Validation loss')
plt.plot(history.history['rmse'], label='RMSE Training loss')
plt.plot(history.history['val_rmse'], label='RMSE Validation loss')
plt.legend();

X_train_pred = model.predict(X_train, verbose=0)
train_mae_loss = np.mean(np.abs(X_train_pred - X_train), axis=1)

plt.hist(train_mae_loss, bins=50)
plt.xlabel('Train MAE loss')
plt.ylabel('Number of Samples');

Validació del model

def evaluate_prediction(predictions, actual, model_name):
    errors = predictions - actual
    mse = np.square(errors).mean()
    rmse = np.sqrt(mse)
    mae = np.abs(errors).mean()

    print(model_name + ':')
    print('Mean Absolute Error: {:.4f}'.format(mae))
    print('Root Mean Square Error: {:.4f}'.format(rmse))
    print('Mean Square Error: {:.4f}'.format(mse))
    print('')
    return mae,rmse,mse

mae,rmse,mse = evaluate_prediction(X_train_pred, X_train,MODEL)

GRU:
Mean Absolute Error: 0.2738
Root Mean Square Error: 0.4880
Mean Square Error: 0.2382

model.save(MODELFILENAME+'.h5')

Testejar el model

#càlcul del threshold de test
def calculate_threshold(X_test, X_test_pred):
    distance = np.sqrt(np.mean(np.square(X_test_pred - X_test),axis=1))
    """Sorting the scores/diffs and using a 0.80 as cutoff value to pick the threshold"""
    distance.sort();
    cut_off = int(0.9 * len(distance));
    threshold = distance[cut_off];
    return threshold

for col in columns:
    print ("####################### "+col +" ###########################")

    #Standardize the test data
    scaler = StandardScaler()
    test_cpy = test.copy()
    
    test[col] = scaler.fit_transform(test[[col]])
    
    #creem seqüencia amb finestra temporal per les dades de test
    X_test1, y_test1 = create_sequences(test[[col]], test[col])
    print(f'Testing shape: {X_test1.shape}')
    
    #evaluem el model
    eval = model.evaluate(X_test1, y_test1)
    print("evaluate: ",eval)
    
    #predim el model
    X_test1_pred = model.predict(X_test1, verbose=0)
    
    
    evaluate_prediction(X_test1_pred, X_test1,MODEL)
    
    #càlcul del mae_loss
    test1_mae_loss = np.mean(np.abs(X_test1_pred - X_test1), axis=1)
    test1_rmse_loss = np.sqrt(np.mean(np.square(X_test1_pred - X_test1),axis=1))

    # reshaping test prediction
    X_test1_predReshape = X_test1_pred.reshape((X_test1_pred.shape[0] * X_test1_pred.shape[1]), X_test1_pred.shape[2])
    
    # reshaping test data
    X_test1Reshape = X_test1.reshape((X_test1.shape[0] * X_test1.shape[1]), X_test1.shape[2])
  
    threshold_test = calculate_threshold(X_test1Reshape,X_test1_predReshape)
    
    test1_score_df = pd.DataFrame(test[TIME_STEPS:])
    test1_score_df['loss'] = test1_rmse_loss.reshape((-1))
    test1_score_df['threshold'] = threshold_test
    test1_score_df['anomaly'] = test1_score_df['loss'] > test1_score_df['threshold']
    test1_score_df[col] = test[TIME_STEPS:][col]

    #gràfic test lost i threshold
    fig = go.Figure()
    fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['loss'], name='Test loss'))
    fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['threshold'], name='Threshold'))
    fig.update_layout(showlegend=True, title='Test loss vs. Threshold')
    fig.show()
    
    #Posem les anomalies en un array
    anomalies1 = test1_score_df.loc[test1_score_df['anomaly'] == True]
    anomalies1.shape
    print('anomalies: ',anomalies1.shape); print();
   
    #Gràfic dels punts i de les anomalíes amb els valors de dades transformades per verificar que la normalització que s'ha fet no distorssiona les dades
    fig = go.Figure()
    fig.add_trace(go.Scatter(x=test1_score_df.index, y=scaler.inverse_transform(test1_score_df[col]), name=col))
    fig.add_trace(go.Scatter(x=anomalies1.index, y=scaler.inverse_transform(anomalies1[col]), mode='markers', name='Anomaly'))
    fig.update_layout(showlegend=True, title='Detected anomalies')
    fig.show()
    
    print ("######################################################")

####################### PM1 ###########################

<ipython-input-17-e1f1d6df3b5c>:8: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  test[col] = scaler.fit_transform(test[[col]])

Testing shape: (636, 144, 1)
20/20 [==============================] - 0s 13ms/step - loss: 0.7191 - mse: 1.0950 - rmse: 0.8249
evaluate:  [0.7190611362457275, 1.0950276851654053, 0.8248701691627502]
GRU:
Mean Absolute Error: 0.2367
Root Mean Square Error: 0.5131
Mean Square Error: 0.2633

anomalies:  (250, 10)

######################################################
####################### PM25 ###########################

<ipython-input-17-e1f1d6df3b5c>:8: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

Testing shape: (636, 144, 1)
20/20 [==============================] - 0s 18ms/step - loss: 0.7574 - mse: 1.1842 - rmse: 0.8637
evaluate:  [0.7574223875999451, 1.1841565370559692, 0.8636749982833862]
GRU:
Mean Absolute Error: 0.2528
Root Mean Square Error: 0.5005
Mean Square Error: 0.2505

anomalies:  (218, 10)

######################################################
####################### PM10 ###########################
Testing shape: (636, 144, 1)

<ipython-input-17-e1f1d6df3b5c>:8: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

20/20 [==============================] - 0s 17ms/step - loss: 0.7814 - mse: 1.2433 - rmse: 0.8887
evaluate:  [0.7813811898231506, 1.243257999420166, 0.8887407779693604]
GRU:
Mean Absolute Error: 0.2648
Root Mean Square Error: 0.4873
Mean Square Error: 0.2374

anomalies:  (104, 10)

######################################################
####################### PM1ATM ###########################

<ipython-input-17-e1f1d6df3b5c>:8: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

Testing shape: (636, 144, 1)
20/20 [==============================] - 0s 18ms/step - loss: 0.7817 - mse: 1.1567 - rmse: 0.8946
evaluate:  [0.7817322015762329, 1.1566815376281738, 0.8946117758750916]
GRU:
Mean Absolute Error: 0.2666
Root Mean Square Error: 0.4806
Mean Square Error: 0.2310

anomalies:  (72, 10)

######################################################
####################### PM25ATM ###########################

<ipython-input-17-e1f1d6df3b5c>:8: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

Testing shape: (636, 144, 1)
20/20 [==============================] - 0s 19ms/step - loss: 0.7730 - mse: 1.1369 - rmse: 0.8844
evaluate:  [0.7729862332344055, 1.1368567943572998, 0.8843849301338196]
GRU:
Mean Absolute Error: 0.2615
Root Mean Square Error: 0.4817
Mean Square Error: 0.2320

anomalies:  (72, 10)

######################################################
####################### PM10ATM ###########################

<ipython-input-17-e1f1d6df3b5c>:8: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

Testing shape: (636, 144, 1)
20/20 [==============================] - 0s 15ms/step - loss: 0.7694 - mse: 1.1373 - rmse: 0.8757
evaluate:  [0.7693621516227722, 1.1373294591903687, 0.8757347464561462]
GRU:
Mean Absolute Error: 0.2609
Root Mean Square Error: 0.4842
Mean Square Error: 0.2345

anomalies:  (83, 10)

######################################################